package crawler;

import java.text.BreakIterator;
import java.util.*;
import java.util.concurrent.CopyOnWriteArrayList;

public class WikiPageScraper {
	private String htmlPage;
	private CopyOnWriteArrayList<String> sentences;
	private CopyOnWriteArrayList<String> paragraphs;
	private CopyOnWriteArrayList<String> tableInfo;
	private String title;
	private final String REMOVE_TAGS = "\\<.*?\\>";
	private final String REMOVE_CITATIONS = "\\[.*?\\]";
	private final String CONTAINS_DIGIT = ".*?[\\d]+.*?";
	private final String NO_BREAK_SPACE = "&#160;";

	public WikiPageScraper(String htmlPage) {
		this.htmlPage = htmlPage;
		this.sentences = new CopyOnWriteArrayList<String>();
		this.paragraphs = new CopyOnWriteArrayList<String>();
		this.title = ripTitle();
		this.tableInfo = new CopyOnWriteArrayList<String>();
		ripTable();
		ripParagraphs();
		ripSentences();

	}

	/**
	 * @author Nicole Becker
	 * @return String of the title of the html page without wikipedia trailing
	 *         title
	 */
	private String ripTitle() {
		// (Anthony Dare) Added in some error checking if the title does not exist for whatever reason.
		if (htmlPage.indexOf("<title>") >= 0
				&& htmlPage.indexOf(" - Wikipedia, the free encyclopedia") >= 0) {
			return htmlPage.substring(htmlPage.indexOf("<title>") + 7,
					htmlPage.indexOf(" - Wikipedia, the free encyclopedia"));
		} else {
			return "";
		}
	}

	/**
	 * @author Nicole Becker
	 * @return CopyOnWriteArrayList<String> of Paragraphs Ripped from the
	 *         htmlPage
	 */
	private void ripParagraphs() {
		String temp = htmlPage;
		while (temp.contains("<p>")) {
			String para = temp.substring(temp.indexOf("<p>"),
					temp.indexOf("</p>"));
			// Clean paragraph
			para = para.replaceAll(REMOVE_TAGS, "");
			para = para.replaceAll(REMOVE_CITATIONS, "");
			para = para.replaceAll(NO_BREAK_SPACE, " ");
			// Add paragraph
			if (!para.isEmpty() && para.matches(CONTAINS_DIGIT)) {
				paragraphs.add(para);
			}

			temp = temp.substring(temp.indexOf("</p>") + 4);
		}
	}

	/**
	 * This method contains modified code by From Julien Silland Source:
	 * http://stackoverflow
	 * .com/questions/2687012/split-string-into-sentences-based-on-periods
	 * 
	 * Modified by Nicole Becker
	 * 
	 * @param CopyOnWriteArrayList
	 *            <String> of Paragraphs ripped from the htmlPage
	 */
	private void ripSentences() {
		ListIterator<String> listIt = paragraphs.listIterator();
		while (listIt.hasNext()) {
			BreakIterator breakIt = BreakIterator
					.getSentenceInstance(Locale.US);
			String source = listIt.next();
			breakIt.setText(source);
			int start = breakIt.first();
			for (int end = breakIt.next(); end != BreakIterator.DONE; start = end, end = breakIt
					.next()) {
				String sentence = source.substring(start, end);
				if (sentence.matches(CONTAINS_DIGIT)) {
					sentences.add(sentence);
				}
			}
		}
	}

	public void ripTable() {
		if (htmlPage.contains("<table")) {
			String table = htmlPage.substring(htmlPage.indexOf("<table"));
			table = table.substring(0, table.indexOf("</table>"));
			// Remove html tags
			String[] s = table.split("\\<.*?\\>");
			for (int i = 0; i < s.length; i++) {
				if (!(s[i].trim().isEmpty()))
					tableInfo.add(s[i]);
			}
		}

	}

	public String getHtmlPage() {
		return this.htmlPage;
	}

	public String getTitle() {
		return this.title;
	}

	public CopyOnWriteArrayList<String> getSentences() {
		return this.sentences;
	}

	public CopyOnWriteArrayList<String> getParagraphs() {
		return this.paragraphs;
	}

	public CopyOnWriteArrayList<String> getTableInfo() {
		return this.tableInfo;
	}
}
